{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "history = 10\n",
    "ac_pitch_window = sorted([-19, -12, 0, 12, 19])\n",
    "la_pitch_window = list(range(-5, 6))\n",
    "num_features = 7 #X.shape[1] - len(ac_pitch_window) * history - history * len(la_pitch_window) - 1\n",
    "\n",
    "acoustic_in = keras.layers.Input(shape=(len(ac_pitch_window) * history,), dtype='float', name='acoustic')\n",
    "language_in = keras.layers.Input(shape=(len(la_pitch_window) * history,), dtype='float', name='language')\n",
    "features_in = keras.layers.Input(shape=(num_features + 1,), dtype='float', name='features')\n",
    "\n",
    "# Acoustic model history - 1d convolutions in each direction\n",
    "acoustic = keras.layers.Reshape((len(ac_pitch_window), history, 1),\n",
    "                                input_shape=(len(ac_pitch_window) * history,))(acoustic_in)\n",
    "ac_num_pitch_convs = 5\n",
    "ac_num_history_convs = 10\n",
    "acoustic = keras.layers.Conv2D(ac_num_pitch_convs, (len(ac_pitch_window), 1))(acoustic)\n",
    "acoustic = keras.layers.Permute((3, 2, 1))(acoustic)\n",
    "acoustic = keras.layers.Conv2D(ac_num_history_convs, (ac_num_pitch_convs, min(history, 5)))(acoustic)\n",
    "acoustic = keras.layers.Flatten()(acoustic)\n",
    "\n",
    "# Language model history - series of 2D convolutions\n",
    "language = keras.layers.Reshape((len(la_pitch_window), history, 1),\n",
    "                                input_shape=(len(la_pitch_window) * history,))(language_in)\n",
    "language = keras.layers.Conv2D(5, (3, 3), strides=1)(language)\n",
    "language = keras.layers.Conv2D(5, (3, 3), strides=2)(language)\n",
    "language = keras.layers.Flatten()(language)\n",
    "\n",
    "# Dense layers\n",
    "dense_in = keras.layers.Concatenate()([acoustic, language, features_in])\n",
    "\n",
    "dense = keras.layers.Dense(20, activation='relu')(dense_in)\n",
    "dense = keras.layers.Dropout(0.2)(dense)\n",
    "\n",
    "dense = keras.layers.Dense(20, activation='relu')(dense)\n",
    "dense = keras.layers.Dropout(0.2)(dense)\n",
    "\n",
    "dense = keras.layers.Dense(20, activation='relu')(dense)\n",
    "dense = keras.layers.Dropout(0.2)(dense)\n",
    "\n",
    "dense_out = keras.layers.Dense(1, activation='sigmoid')(dense)\n",
    "\n",
    "model = keras.models.Model(inputs=[acoustic_in, language_in, features_in], outputs=dense_out)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "10000/10000 [==============================] - 2s 195us/step - loss: 0.6942 - acc: 0.5079\n",
      "Epoch 2/100\n",
      "10000/10000 [==============================] - 1s 124us/step - loss: 0.6934 - acc: 0.5005\n",
      "Epoch 3/100\n",
      "10000/10000 [==============================] - 1s 125us/step - loss: 0.6932 - acc: 0.4997\n",
      "Epoch 4/100\n",
      "10000/10000 [==============================] - 1s 126us/step - loss: 0.6928 - acc: 0.5098\n",
      "Epoch 5/100\n",
      "10000/10000 [==============================] - 1s 126us/step - loss: 0.6928 - acc: 0.5069\n",
      "Epoch 6/100\n",
      "10000/10000 [==============================] - 1s 125us/step - loss: 0.6934 - acc: 0.5063\n",
      "Epoch 7/100\n",
      "10000/10000 [==============================] - 1s 127us/step - loss: 0.6928 - acc: 0.5127\n",
      "Epoch 8/100\n",
      "10000/10000 [==============================] - 1s 126us/step - loss: 0.6917 - acc: 0.5185\n",
      "Epoch 9/100\n",
      "10000/10000 [==============================] - 1s 127us/step - loss: 0.6897 - acc: 0.5271\n",
      "Epoch 10/100\n",
      "10000/10000 [==============================] - 1s 128us/step - loss: 0.6903 - acc: 0.5303\n",
      "Epoch 11/100\n",
      "10000/10000 [==============================] - 1s 128us/step - loss: 0.6884 - acc: 0.5373\n",
      "Epoch 12/100\n",
      "10000/10000 [==============================] - 1s 134us/step - loss: 0.6878 - acc: 0.5381\n",
      "Epoch 13/100\n",
      "10000/10000 [==============================] - 1s 127us/step - loss: 0.6845 - acc: 0.5484\n",
      "Epoch 14/100\n",
      "10000/10000 [==============================] - 1s 127us/step - loss: 0.6841 - acc: 0.5537\n",
      "Epoch 15/100\n",
      "10000/10000 [==============================] - 1s 128us/step - loss: 0.6803 - acc: 0.5574\n",
      "Epoch 16/100\n",
      "10000/10000 [==============================] - 1s 126us/step - loss: 0.6791 - acc: 0.5591\n",
      "Epoch 17/100\n",
      "10000/10000 [==============================] - 1s 136us/step - loss: 0.6771 - acc: 0.5722\n",
      "Epoch 18/100\n",
      "10000/10000 [==============================] - 1s 133us/step - loss: 0.6740 - acc: 0.5750\n",
      "Epoch 19/100\n",
      "10000/10000 [==============================] - 1s 131us/step - loss: 0.6719 - acc: 0.5813\n",
      "Epoch 20/100\n",
      "10000/10000 [==============================] - 1s 132us/step - loss: 0.6679 - acc: 0.5803\n",
      "Epoch 21/100\n",
      "10000/10000 [==============================] - 1s 129us/step - loss: 0.6654 - acc: 0.5876\n",
      "Epoch 22/100\n",
      "10000/10000 [==============================] - 1s 134us/step - loss: 0.6638 - acc: 0.5931\n",
      "Epoch 23/100\n",
      "10000/10000 [==============================] - 1s 132us/step - loss: 0.6594 - acc: 0.5970\n",
      "Epoch 24/100\n",
      "10000/10000 [==============================] - 1s 134us/step - loss: 0.6558 - acc: 0.5997\n",
      "Epoch 25/100\n",
      "10000/10000 [==============================] - 1s 132us/step - loss: 0.6538 - acc: 0.6047\n",
      "Epoch 26/100\n",
      "10000/10000 [==============================] - 1s 134us/step - loss: 0.6482 - acc: 0.6083\n",
      "Epoch 27/100\n",
      "10000/10000 [==============================] - 1s 133us/step - loss: 0.6466 - acc: 0.6113\n",
      "Epoch 28/100\n",
      "10000/10000 [==============================] - 1s 128us/step - loss: 0.6411 - acc: 0.6225\n",
      "Epoch 29/100\n",
      "10000/10000 [==============================] - 1s 129us/step - loss: 0.6417 - acc: 0.6189\n",
      "Epoch 30/100\n",
      "10000/10000 [==============================] - 1s 128us/step - loss: 0.6382 - acc: 0.6217\n",
      "Epoch 31/100\n",
      "10000/10000 [==============================] - 1s 128us/step - loss: 0.6383 - acc: 0.6196\n",
      "Epoch 32/100\n",
      "10000/10000 [==============================] - 1s 130us/step - loss: 0.6371 - acc: 0.6225\n",
      "Epoch 33/100\n",
      "10000/10000 [==============================] - 1s 130us/step - loss: 0.6311 - acc: 0.6283\n",
      "Epoch 34/100\n",
      "10000/10000 [==============================] - 1s 129us/step - loss: 0.6286 - acc: 0.6326\n",
      "Epoch 35/100\n",
      "10000/10000 [==============================] - 1s 131us/step - loss: 0.6311 - acc: 0.6272\n",
      "Epoch 36/100\n",
      "10000/10000 [==============================] - 1s 130us/step - loss: 0.6276 - acc: 0.6347\n",
      "Epoch 37/100\n",
      "10000/10000 [==============================] - 1s 130us/step - loss: 0.6219 - acc: 0.6322\n",
      "Epoch 38/100\n",
      "10000/10000 [==============================] - 1s 130us/step - loss: 0.6223 - acc: 0.6400\n",
      "Epoch 39/100\n",
      "10000/10000 [==============================] - 1s 131us/step - loss: 0.6197 - acc: 0.6408\n",
      "Epoch 40/100\n",
      "10000/10000 [==============================] - 1s 129us/step - loss: 0.6170 - acc: 0.6467\n",
      "Epoch 41/100\n",
      "10000/10000 [==============================] - 1s 129us/step - loss: 0.6219 - acc: 0.6380\n",
      "Epoch 42/100\n",
      "10000/10000 [==============================] - 1s 130us/step - loss: 0.6146 - acc: 0.6437\n",
      "Epoch 43/100\n",
      "10000/10000 [==============================] - 1s 130us/step - loss: 0.6168 - acc: 0.6436\n",
      "Epoch 44/100\n",
      "10000/10000 [==============================] - 1s 130us/step - loss: 0.6150 - acc: 0.6502\n",
      "Epoch 45/100\n",
      "10000/10000 [==============================] - 1s 131us/step - loss: 0.6102 - acc: 0.6468\n",
      "Epoch 46/100\n",
      "10000/10000 [==============================] - 1s 130us/step - loss: 0.6156 - acc: 0.6422\n",
      "Epoch 47/100\n",
      "10000/10000 [==============================] - 1s 133us/step - loss: 0.6071 - acc: 0.6507\n",
      "Epoch 48/100\n",
      "10000/10000 [==============================] - 1s 131us/step - loss: 0.6078 - acc: 0.6535\n",
      "Epoch 49/100\n",
      "10000/10000 [==============================] - 1s 131us/step - loss: 0.6022 - acc: 0.6545\n",
      "Epoch 50/100\n",
      "10000/10000 [==============================] - 1s 131us/step - loss: 0.6033 - acc: 0.6534\n",
      "Epoch 51/100\n",
      "10000/10000 [==============================] - 1s 132us/step - loss: 0.6040 - acc: 0.6531\n",
      "Epoch 52/100\n",
      "10000/10000 [==============================] - 1s 130us/step - loss: 0.6049 - acc: 0.6527\n",
      "Epoch 53/100\n",
      "10000/10000 [==============================] - 1s 127us/step - loss: 0.5978 - acc: 0.6585\n",
      "Epoch 54/100\n",
      "10000/10000 [==============================] - 1s 127us/step - loss: 0.6008 - acc: 0.6586\n",
      "Epoch 55/100\n",
      "10000/10000 [==============================] - 1s 127us/step - loss: 0.5982 - acc: 0.6584\n",
      "Epoch 56/100\n",
      "10000/10000 [==============================] - 1s 127us/step - loss: 0.5975 - acc: 0.6606\n",
      "Epoch 57/100\n",
      "10000/10000 [==============================] - 1s 126us/step - loss: 0.5871 - acc: 0.6704\n",
      "Epoch 58/100\n",
      "10000/10000 [==============================] - 1s 126us/step - loss: 0.5965 - acc: 0.6607\n",
      "Epoch 59/100\n",
      "10000/10000 [==============================] - 1s 127us/step - loss: 0.5920 - acc: 0.6638\n",
      "Epoch 60/100\n",
      "10000/10000 [==============================] - 1s 127us/step - loss: 0.5923 - acc: 0.6671\n",
      "Epoch 61/100\n",
      "10000/10000 [==============================] - 1s 127us/step - loss: 0.5902 - acc: 0.6606\n",
      "Epoch 62/100\n",
      "10000/10000 [==============================] - 1s 126us/step - loss: 0.5923 - acc: 0.6652\n",
      "Epoch 63/100\n",
      "10000/10000 [==============================] - 1s 127us/step - loss: 0.5867 - acc: 0.6701\n",
      "Epoch 64/100\n",
      "10000/10000 [==============================] - 1s 126us/step - loss: 0.5927 - acc: 0.6668\n",
      "Epoch 65/100\n",
      "10000/10000 [==============================] - 1s 126us/step - loss: 0.5852 - acc: 0.6700\n",
      "Epoch 66/100\n",
      "10000/10000 [==============================] - 1s 127us/step - loss: 0.5863 - acc: 0.6736\n",
      "Epoch 67/100\n",
      "10000/10000 [==============================] - 1s 126us/step - loss: 0.5817 - acc: 0.6719\n",
      "Epoch 68/100\n",
      "10000/10000 [==============================] - 1s 127us/step - loss: 0.5890 - acc: 0.6679\n",
      "Epoch 69/100\n",
      "10000/10000 [==============================] - 1s 126us/step - loss: 0.5843 - acc: 0.6741\n",
      "Epoch 70/100\n",
      "10000/10000 [==============================] - 1s 127us/step - loss: 0.5856 - acc: 0.6724\n",
      "Epoch 71/100\n",
      "10000/10000 [==============================] - 1s 126us/step - loss: 0.5846 - acc: 0.6714\n",
      "Epoch 72/100\n",
      "10000/10000 [==============================] - 1s 127us/step - loss: 0.5833 - acc: 0.6709\n",
      "Epoch 73/100\n",
      "10000/10000 [==============================] - 1s 137us/step - loss: 0.5801 - acc: 0.6761\n",
      "Epoch 74/100\n",
      "10000/10000 [==============================] - 1s 129us/step - loss: 0.5762 - acc: 0.6827\n",
      "Epoch 75/100\n",
      "10000/10000 [==============================] - 1s 130us/step - loss: 0.5829 - acc: 0.6685\n",
      "Epoch 76/100\n",
      "10000/10000 [==============================] - 1s 130us/step - loss: 0.5838 - acc: 0.6673\n",
      "Epoch 77/100\n",
      "10000/10000 [==============================] - 1s 130us/step - loss: 0.5730 - acc: 0.6760\n",
      "Epoch 78/100\n",
      "10000/10000 [==============================] - 1s 130us/step - loss: 0.5796 - acc: 0.6758\n",
      "Epoch 79/100\n",
      "10000/10000 [==============================] - 1s 130us/step - loss: 0.5770 - acc: 0.6789\n",
      "Epoch 80/100\n",
      "10000/10000 [==============================] - 1s 127us/step - loss: 0.5787 - acc: 0.6769\n",
      "Epoch 81/100\n",
      "10000/10000 [==============================] - 1s 127us/step - loss: 0.5744 - acc: 0.6768\n",
      "Epoch 82/100\n",
      "10000/10000 [==============================] - 1s 127us/step - loss: 0.5716 - acc: 0.6791\n",
      "Epoch 83/100\n",
      "10000/10000 [==============================] - 1s 129us/step - loss: 0.5697 - acc: 0.6836\n",
      "Epoch 84/100\n",
      "10000/10000 [==============================] - 1s 125us/step - loss: 0.5692 - acc: 0.6809\n",
      "Epoch 85/100\n",
      "10000/10000 [==============================] - 1s 126us/step - loss: 0.5691 - acc: 0.6850\n",
      "Epoch 86/100\n",
      "10000/10000 [==============================] - 1s 125us/step - loss: 0.5705 - acc: 0.6773\n",
      "Epoch 87/100\n",
      "10000/10000 [==============================] - 1s 126us/step - loss: 0.5683 - acc: 0.6827\n",
      "Epoch 88/100\n",
      "10000/10000 [==============================] - 1s 125us/step - loss: 0.5718 - acc: 0.6763\n",
      "Epoch 89/100\n",
      "10000/10000 [==============================] - 1s 126us/step - loss: 0.5686 - acc: 0.6830\n",
      "Epoch 90/100\n",
      "10000/10000 [==============================] - 1s 125us/step - loss: 0.5700 - acc: 0.6818\n",
      "Epoch 91/100\n",
      "10000/10000 [==============================] - 1s 126us/step - loss: 0.5683 - acc: 0.6862\n",
      "Epoch 92/100\n",
      "10000/10000 [==============================] - 1s 125us/step - loss: 0.5683 - acc: 0.6815\n",
      "Epoch 93/100\n",
      "10000/10000 [==============================] - 1s 126us/step - loss: 0.5686 - acc: 0.6864\n",
      "Epoch 94/100\n",
      "10000/10000 [==============================] - 1s 125us/step - loss: 0.5645 - acc: 0.6853\n",
      "Epoch 95/100\n",
      "10000/10000 [==============================] - 1s 125us/step - loss: 0.5704 - acc: 0.6895\n",
      "Epoch 96/100\n",
      "10000/10000 [==============================] - 1s 125us/step - loss: 0.5670 - acc: 0.6853\n",
      "Epoch 97/100\n",
      "10000/10000 [==============================] - 1s 126us/step - loss: 0.5631 - acc: 0.6897\n",
      "Epoch 98/100\n",
      "10000/10000 [==============================] - 1s 126us/step - loss: 0.5667 - acc: 0.6841\n",
      "Epoch 99/100\n",
      "10000/10000 [==============================] - 1s 125us/step - loss: 0.5596 - acc: 0.6874\n",
      "Epoch 100/100\n",
      "10000/10000 [==============================] - 1s 125us/step - loss: 0.5658 - acc: 0.6921\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f14c2909978>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = np.random.rand(10000, len(ac_pitch_window) * history + history * len(la_pitch_window) + 1 + 7)\n",
    "Y = np.round(np.random.rand(10000))\n",
    "\n",
    "acoustic_in = X[:, :len(ac_pitch_window) * history]\n",
    "language_in = X[:, len(ac_pitch_window) * history:history * (len(ac_pitch_window) + len(la_pitch_window))]\n",
    "features_in = X[:, history * (len(ac_pitch_window) + len(la_pitch_window)):]\n",
    "\n",
    "model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "model.fit([acoustic_in, language_in, features_in], Y, epochs=100, batch_size=32)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
